#!/usr/bin/env python3

"""multiproc.py - Methods for distributing wiki work among multiple processes
"""

import os
import multiprocessing as mp
from optparse import OptionParser
import wikiproc

def extract_translations(languages, pool, files):
    print(pool.map(tr, [(languages, file) for file in files]))

def tr(tpl):
    """ translate a tuple
    
    Because pool's version of map can only take one iterably argument,
    we define a utility function here. Note that this function must have
    module-wide scope. Defining it within the scope of extract_translations
    causes a pickle error.
    """
    return translate(tpl[0], tpl[1])

def translate(languages, file):
    outfile = file + ".json"
    dump = wikiproc.WikiDump(file, languages, proc=True)
    with open(outfile, 'w') as out:
       out.write(wikiproc.json_encoder().encode(dump)) 
    return outfile

def main():
    parser = OptionParser()
    parser.add_option("-t", "--translate",
                      action="append", dest="targetlangs", 
                      help="translate wiki file into target language in JSON")
    parser.add_option("-n", "--num-processes",
                      type="int", dest="num_processes", default=mp.cpu_count(),
                      help="number of processes to use")
    (options, args) = parser.parse_args()

    pool = mp.Pool(options.num_processes)

    if (options.targetlangs):
        extract_translations(options.targetlangs, pool, args)

if __name__ == '__main__':
    exit(main())
